# coding:utf8
import url_manager, html_downloader, url_parser, html_outputer

class SpiderMain(object):
    def __init__(self):
        self.urls = url_manager.UrlManager()  # url管理器类
        self.downloader = html_downloader.HtmlDownloader()  # html下载器类
        self.parser = url_parser.HtmlParser()  # url解析器类
        self.outputer = html_outputer.HtmlOutputer()  # html输出类

    def craw(self, root_url,book_type,max_num):
        self.urls.add_new_url(root_url)  # 第一次运行爬虫时先把首个url加入url管理器
        print "Star crawing:"+root_url[0]
        count=1
        while self.urls.has_new_url():  # 当url管理器里没有新的url就结束爬虫
            try:
                new_url = self.urls.get_new_url()  # 获取新的url
                html_cont,getcode = self.downloader.download(new_url)  # 下载目标html
                linklist, booklist = self.parser.parser_url(root_url, html_cont)  # 把下载好的html已经当前url放入解释器 返回新的url和筛选出来的数据
                self.urls.add_new_url(linklist)  # 把新的url添加到url管理器中
                self.urls.add_new_book(booklist)
                while self.urls.has_new_book():
                    try:
                        new_book = self.urls.get_new_book()
                        html_cont,getcode = self.downloader.download(new_book)
                        book_data = self.parser.parser_content(html_cont)
                        self.outputer.collect_data(book_data,book_type)
                    except:
                        print new_book
                        print getcode
                        print "craw book_page fail"
                print new_url+"  已爬"
            except:
                print new_url
                print getcode
                print "craw index_page fail"
            if count == max_num:
                break
            count+=1
        self.outputer.output_html()  # 输出
if __name__ == "__main__":
    all_root_url={
            1:"https://www.douban.com/tag/%E7%A7%91%E6%99%AE/book",         #科普
            2:"https://www.douban.com/tag/%E7%BC%96%E7%A8%8B/book",         #编程
            3:"https://www.douban.com/tag/UCD/book",                        #UCD
            4:"https://www.douban.com/tag/web/book",                        #web
            5:"https://www.douban.com/tag/%E7%94%A8%E6%88%B7%E4%BD%93%E9%AA%8C/book",       #用户体验
            6:"https://www.douban.com/tag/%E7%A7%91%E5%B9%BB/book",                         #科幻小说
            7:"https://www.douban.com/tag/%E8%BD%BB%E5%B0%8F%E8%AF%B4/book",                #轻小说
            8:"https://www.douban.com/tag/%E6%82%AC%E7%96%91/book",                         #悬疑
            9:"https://www.douban.com/tag/%E6%8E%A8%E7%90%86%E5%B0%8F%E8%AF%B4/book",        #推理小说

    }
    select=("科普","编程","UCD","WEB","用户体验","科幻小说","轻小说","悬疑","推理小说")
    for index,name in enumerate(select):
        print (index+1),':',unicode(name)
    print u"请选择你要爬取书种(序号):"
    select_num=input()
    print u"请输入要爬取的页数:"
    max_num=input()
    root_url = [all_root_url[select_num]]
    obj_spider = SpiderMain()
    obj_spider.craw(root_url,select[select_num-1],max_num)
